{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "import numpy as np\n",
    "\n",
    "import os\n",
    "import wandb\n",
    "from wandb.keras import WandbCallback\n",
    "from PIL import Image\n",
    "from keras.callbacks import ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W&B Run: https://app.wandb.ai/qualcomm/Distillation/runs/jx4egmee\n",
      "Call `%%wandb` in the cell containing your training loop to display live results.\n"
     ]
    }
   ],
   "source": [
    "run = wandb.init(project=\"Distillation\", tensorboard=True)\n",
    "config = run.config\n",
    "config.dropout = 0.25\n",
    "config.dense_layer_nodes = 100\n",
    "config.learn_rate = 0.32  #0.01 \n",
    "config.batch_size = 1024  #32\n",
    "config.epochs = 100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = ['airplane','automobile','bird','cat','deer',\n",
    "               'dog','frog','horse','ship','truck']\n",
    "num_classes = len(class_names)\n",
    "\n",
    "(X_train, y_train), (X_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "# Normalize values into [0 to 1] because all default assume zero to one\n",
    "# We won't bother to fully sphere the data, removing median and divide by variance\n",
    "X_train = X_train.astype('float32') / 255.\n",
    "X_test = X_test.astype('float32') / 255.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 32, 32, 3)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 1., 0., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH3VJREFUeJztnVuMXNd1pv9Vt67qezf7QrJJiRJ1ieRYomRG0MiejB0jgWIEkQ0Ejv1g6MEIgyAGYiB5EDzA2APMgz0Y2/DDwAN6pEQZeHyJL7EQCEkcwYGQOFBEWbLukSiKMi/NZpPdze7qqq7rmocqTaj2/jdLvFRT2v8HEKw+q/Y56+w665w656+1lrk7hBDpkdlqB4QQW4OCX4hEUfALkSgKfiESRcEvRKIo+IVIFAW/EImi4BciURT8QiRK7lIGm9m9AL4GIAvgf7v7F2Pvz+fzPlAsBm2tVouOyyD8K8Ss8W0Vcvy8lo/YctkstZmFN2gWOYdGfGw2+T7HfneZjflIfrHZ9jbfVptvzTKRHYjQbof3LeZ7dH0R/y0yycyWifiRzfDPkx0DANCO/FrWYwcCGxNdX5illTWUKxs9beyig9/MsgD+J4DfBHAcwJNm9oi7v8jGDBSL2Hfn+4K2lZUluq2BTPiDnyzwyblm2yC1TU8OUdvU+DC1FbL54PLcQImOQZZP8dLyCrXVm3zfJsbHqC3TagSX12o1OmZjY4PaiqXwyRoAWuAnr0q1HFw+Nj5Kx8D5+uq1OrVlEf5cAH6yGRnmn/PQED8+8nk+H9WIjx67QGTCx0hsn5seju8vPfh9vp3Nm+35nb/MXQAOu/sRd68D+DaA+y5hfUKIPnIpwT8H4Nh5fx/vLhNCvAO4pHv+XjCzAwAOAMDAwMCV3pwQokcu5cp/AsDu8/7e1V32Ftz9oLvvd/f9uTy/NxNC9JdLCf4nAdxoZteZWQHAJwA8cnncEkJcaS76a7+7N83sMwD+Dh2p7yF3fyE2ZmNjAy+8GH7LypkzdNwkecBq2/iT16nWCLVZaYba1ttcdSi3wk/g3Qp0TGWDP7GtVPkT+EaLS1tnIhpnMRf2sdnk68uSp81A/FatsrFObc12eL9tYxsdk4mogI2IWlHK8eOgTJ6YL7WadMzgIH/abxn+7dWIGgQAiMiHlY2wQtNshJcDQDYX/lwaG1XuwyYu6Z7f3R8F8OilrEMIsTXoF35CJIqCX4hEUfALkSgKfiESRcEvRKJc8V/4nU8GQClHZKrIj/+uJZLenlme4DIzPUltpZiUE8naqtbCCTAbDS5DeWR9hVIkISiS2ONtvr2xyXBCU7PB11fIcz8iyZbIFviHVquH56rR5PMxGFlfboj7WIyMa1pYjsxEsgSbkQy8WCbp8BBPJiuvV6it0QxLerGEyrXVc8Hl7dgHtnn9Pb9TCPGuQsEvRKIo+IVIFAW/EImi4BciUfr6tN/MUbRwQsXICHflprmJ4PJtJZ4Jkm/z0lTlJZ5s02rz82G1EvY9w/N6MBopC5aLPKVeObfGx0U+tcmR8BPntVWehFOPJOhUSdIJEK9LN0xKYTXqPPEk0+I7lo8kGLVI6TIAyJHH87UaH1PI8w800+YJQbXyMrWBJIUBwAA5jJttrkicWw8rPq1IPcbN6MovRKIo+IVIFAW/EImi4BciURT8QiSKgl+IROmr1Jczw8RAeJOliJQzRpI6pkd5zbQWaRcFINJnBsjmIoXkSB22WjsiNUV0uVwkuaRV45KYZ/k5+/TpcBegVoPv9VqFJ51UWlwWHS5Fuu/USLsu8H3OGJepsgORTjnrXNYdzId9zEVaYW1E6i5WG1zqa0earK2UuY8rlfDxUybSMgBsNMLHQD1Sq3EzuvILkSgKfiESRcEvRKIo+IVIFAW/EImi4BciUS5J6jOzowDW0FHPmu6+P7qxrGF6PCzZjOS5xFYshm2ZLJdWSpH6eI0ml73akUw197AEVI/U22vVuQzY9kjGXERi8xzPOlurhzP0Wi0+v5VIa7BmxLa2zv0/sRT2I5/h6xst87lvnOLt3KrnuFR5zdQNweUzM7voGBsJ18cDgNryWWorl3l25Lk1LvWdOReWdY8e4360suHQrdW5PLiZy6Hzf8jd+ScjhLgq0dd+IRLlUoPfAfy9mT1lZgcuh0NCiP5wqV/7P+DuJ8xsBsCPzexld3/8/Dd0TwoHAKAYua8XQvSXS7ryu/uJ7v+nAfwQwF2B9xx09/3uvr+Q012GEFcLFx2NZjZkZiNvvgbwWwCev1yOCSGuLJfytX8WwA+77a1yAP6vu/9tbEA+l8XO6XBhx9EClyiGB8PSlkWkMkQyrCySTVerctkoQ2TAbSO8bdjQEM9GWz3HRZKxUZ4xtxYpqvnGifA6yzV+y1WIJILNDUayEvM88/Do2XB2Yc0jRVcjWX1joyPUds+tXGFenQ/Lul6JbGuKZ4vWKnw+ymV+LR3I83Xu3h7et5mZWTpmYTUsHZ595RQds5mLDn53PwLg9osdL4TYWnQTLkSiKPiFSBQFvxCJouAXIlEU/EIkSn8LeGYNkyPhbLtcPSwNAcBAPuzm4EC4Lx0A1KpcDmtE+q2Nj4f7AgKAk6KP9RY/hzYakeKSw7yP38nFcC82AHjtDZ7ttbgW3rdILUhcG+l5+NH/uI/adu3g/n/vqSPB5f9ymEtRzTbPZMxluDS3trJIbZVyeB5HRrj0hhbPLiwW+bgCyT4FgEHj45qt8Idzze6ddMzIUriX47Ov87nYjK78QiSKgl+IRFHwC5EoCn4hEkXBL0Si9Pdpfy6HmcltQVt1iT8Vz1jYzTJpcwQA1Ugts5xF6tlF2lqxM2W1wZ9Sj0/wBJ16iz/BPnL8JLUtrXIfWX2/bKTF12iRr28mF36qDADFJa5I3Di6Pbh8fpL7sbBymtpqFT7HT7/yCrVlSPuqxlCk1dgYT6hBhofM2BhXn0bakfZgpM6j11fpmD0kQW4g3/v1XFd+IRJFwS9Eoij4hUgUBb8QiaLgFyJRFPxCJEqfpb48Jqamg7aJYd5eK5MJJ0WsrC7TMY31Ml9fK9auixe0c5JgNDzM6/Q1wG0vHeES1XqNt34qFge4rRD2sTTEZaiJLJdFnzq8QG3NOj98amNhqW96gs+HgctvjSaXgit1XktwndTqqzf5PltEuo10c0M+E2n1lonULsyF57FZ41KqE5mY5J4F0ZVfiERR8AuRKAp+IRJFwS9Eoij4hUgUBb8QiXJBqc/MHgLwOwBOu/uvdpdNAvgOgD0AjgL4uLtz3e3f1wYQ2c4i7YwYA5F6aoMIZz0BQC5yzstkIvX4iAw4UOLtus6c4llxlTN8yq6f5JJYjateKBJJ7+a9c3RMJrLCZpbP8WpEas1lw3UGRwr8c9k2sZfa9t54DbW9/osnqe3lV04ElxdyERnNuUzcbPKQyZCMSgDIF/g8ttvh46od0RXNwsdpRIn8JXq58v8FgHs3LXsAwGPufiOAx7p/CyHeQVww+N39cQBLmxbfB+Dh7uuHAXz0MvslhLjCXOw9/6y7z3dfn0KnY68Q4h3EJT/w804xe/qjQjM7YGaHzOzQWiVysyqE6CsXG/wLZrYDALr/0/pL7n7Q3fe7+/6RQf4QSwjRXy42+B8BcH/39f0AfnR53BFC9ItepL5vAfgggCkzOw7g8wC+COC7ZvZpAG8A+HgvG2u7o7oRLlZoDZ6ZBYQzsNbXeYHDeoOf15oZ/g2kXOHS3Cqxze3m0+hNvr5rp7gws3cnl4YqG3zc3E23B5cXnN9yLZ/jhVBL4+GCqwCAszxTbff2HcHlK+s8W/H6X7mR2kYneFbi6MQt1La8GJ7/5XO85Vk+IkdmnGdUNtqRbFGeLIpWI3x8R5IEaeu4t5HUd+Hgd/dPEtOH38Z2hBBXGfqFnxCJouAXIlEU/EIkioJfiERR8AuRKH0t4OlwtCwsh3iLF1RkskapyIt+Do9waejkIpcVXz++SG25fNiPwgLvq7exwNd34wyX8z78QS57vXZic6rFvzMyFy6QOrUtXFATAE4v8iKd4+MR2avN/S+QgpWnF8NZdgCQK65Q2+LKPLWdmOdZePl8+DgYH+XaW7XKBTPP8eulRbS5dkQGzFh4nEUyTCNtHntGV34hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkSl+lvmw2g/Hx4aCtmeNSX7kczkjzBpdPzq3xrK03fsGlrXKZy0alYvhcOf86zy6cLfKijnNz11Lb+M7rqC2/FkkRI0VNd91+Fx9yistvpSaXKlvgmYLr62HbjsGwFAkA9RbfLxsKHzcAsGtoJ7WNjIclzrWzp+iY0wtnqa1hXN7cqPOioMhwbW5oIJxlWq9GJExSENSIbBh0qed3CiHeVSj4hUgUBb8QiaLgFyJRFPxCJEpfn/a3W02srYSfpObqvNZdnrQmAi8hh1yWGytlrgRMjPBElvGh8FPZ6jJ/2j+zk9fAm7vtP1Hb88fr1PbKYW67Z8dkcPnKCh8zuzdc9w8AMqhQW73GlYBxDz+5Xz3Nn6SX6ryW4I7J8H4BwEqL19XL3zYRXF6NJAr986OPUNvxY3yfs5GWXLFGWiyPqBFrK9cIzxVLgguuo+d3CiHeVSj4hUgUBb8QiaLgFyJRFPxCJIqCX4hE6aVd10MAfgfAaXf/1e6yLwD4AwBv6h6fc/dHe9lgligerUgSgxOZJEPaeAFAy7jUt8wVJayuRuq31cJy2Y4xLg/+2oc+RG27br6b2n7w5w9R2/ZIkku2Hq5PeOLIa3x9199KbcVtN1DbkHN5trIU7t1aaoelNwCoV7mseGaN28aneRLUtu17gsur5VE6JsNNaBV4MlOshl+jwaVWa4YT1Mx54lqzGQ7dyy31/QWAewPLv+ru+7r/egp8IcTVwwWD390fB8DLxQoh3pFcyj3/Z8zsWTN7yMz4dzkhxFXJxQb/1wHsBbAPwDyAL7M3mtkBMztkZofKFX7fI4ToLxcV/O6+4O4td28D+AYAWibG3Q+6+3533z88yKvaCCH6y0UFv5ntOO/PjwF4/vK4I4ToF71Ifd8C8EEAU2Z2HMDnAXzQzPYBcABHAfxhLxszAEaUiBbJUgJ426JI5yR4NbK+SAm8yW28zdf2wbC0eOf+m+iYW+7hct7yaS5vDjR55uH1u3ZRW5vs3PYZXjuvucEl00okG7De5OMa1fCh1QKXKV87cZzannv+ELXdczf3cdv2cFbl6lpYigQA0uELADC1h8u67Vh7rXpEtiMS8rlF3r6sthZ2sk2yKUNcMPjd/ZOBxQ/2vAUhxFWJfuEnRKIo+IVIFAW/EImi4BciURT8QiRKXwt4ugNtksFUrXGJokCy2HI5XjAxm+Hyzw3b+a+RiyV+Ptxz7e7g8ts/wDP3dtx8G7U98y9/Tm3X7OY+bn/Pe6mtML03uDw3OEbHVDa45Fhd5Zl7CyePUdvyQli2azV4dl5pJFwgFQCmpvhnfezk09Q2u2MuuLxZiWSRVnnbLVtfpraWhzMqAcCZxg2gNBDet8J2vs+rAyTT9W1EtK78QiSKgl+IRFHwC5EoCn4hEkXBL0SiKPiFSJS+Sn1mhnw2vMnlSIHG1kZY1igNluiYbIZLKzORzL1j8zyTau+doVKGwK73hpd34JJdY22d2sZGuDQ3fdM+alvPhXvavfD0k3RMrcr9WF3l83HmxC+oLdsKS63FIj/k5q4Ly3IAcNtNvJBoM8sz7fLZ8fDyAs/6zG3wIp2VN05QG5OxAaAZucyWSV/JwW18v2ZJD8h8vvfrua78QiSKgl+IRFHwC5EoCn4hEkXBL0Si9Dexp91GrRp+kjo4wF2xYvhpaD7Da8h5i9tKw7yV1+/+/u9S2z2//eHg8tGpWTpm4chL1JaN+L+yxmv4LR79N2o7uRZ+4vyPf/3XdMxwiSeQbNR4Asz2Wa5IjI6En1S/fpwnA9Uj8zG5cw+13fTe91EbWgPBxUsrvF5ghahLALBc5T6a82N4o8oT18qkxZaXuepwS1jEQLv3bl268guRKgp+IRJFwS9Eoij4hUgUBb8QiaLgFyJRemnXtRvAXwKYRac910F3/5qZTQL4DoA96LTs+ri78wJnAByOtpPaem2eFGHNsEzS9EhLrkjNtOLAKLXtex+XjQbyYUnsxWd4Dbnlk69RW63GpZy15SVqO3b4RWorezjZKd/i2xrOcelztMiTS6YnuNQ3v3AquLwZactWWeOy4rHXeRIR8AK1lMvhGoTFHD8+mgMz1Ha2yY+dUonXIBwc4UlopVxYjlyrrNIxzXZYcnwbSl9PV/4mgD9191sB3A3gj83sVgAPAHjM3W8E8Fj3byHEO4QLBr+7z7v7z7qv1wC8BGAOwH0AHu6+7WEAH71STgohLj9v657fzPYAuAPAEwBm3X2+azqFzm2BEOIdQs/Bb2bDAL4P4LPu/pabEXd3kNsNMztgZofM7NB6ldfSF0L0l56C38zy6AT+N939B93FC2a2o2vfASDY8NzdD7r7fnffP1QqXA6fhRCXgQsGv5kZgAcBvOTuXznP9AiA+7uv7wfwo8vvnhDiStFLVt/7AXwKwHNm9kx32ecAfBHAd83s0wDeAPDxC6/KAYRlu3aT3xLk8uGae61IzbQ6ePbV7Bivq/d3j/wNtU3OhiWlmR3hNl4AUK/w7Lx8PizxAMDwEJeUchkuzQ0ROXL7TLjmGwBU17hCW8pyH88unqG2Rj382YwUueRVL3Op79WnD1Hb/MuvUFutSVpo5fkctmLzu4tLnxjix3BmgEutRSLbTYDP1S3vuS64vFQ8Qsds5oLB7+7/BIDlOIZzXIUQVz36hZ8QiaLgFyJRFPxCJIqCX4hEUfALkSh9LeAJN7TbYeGgEMksK+ZI8cMML7TokRZO7TrPLDtzJpyNBgDlxbCt1ODZV23w/Zqc4PLb+M5pamu2atR24mTYR4/ke2Uy/DCoN7lkmjVe+HOoGJZnSYJmZ30xYyRLs1XncmqGHG+rFS5v1geIPAhgZCef+/USb2221uYy4MZ6+Bq8bfR6OmaKSLe5fO8hrSu/EImi4BciURT8QiSKgl+IRFHwC5EoCn4hEqW/Uh8MGQtniRUHeAaTkwy9oVJYTgKAoZEpaqs0eIbVthFecyBH/KifW6Bj2hm+vkqeS1uzs+GsLQBo17lsdPNtu4LLf/qTx+iYuleoLW9cTq2W+bjRkXBWYiHHD7msRfrZbfDP7PV5LtutrIQ/s5qt0zHTN/Fr4tx4JCvR+We9fIbPVWEjLJkOzUUyMSvhrMl2RC3djK78QiSKgl+IRFHwC5EoCn4hEkXBL0Si9PVpf8aAQi58vqnUeMJElrSMakfqy1UaPDkjm+dJIgMF/jQ3nw/7URjkbavGRnmC0alFrhJU5sJP7QFgZvcN1HbidLiu3nt+7f10THnxJLUdeYW3wlov80SWXDY8/2NjvDahkfqOADB/gvv4izciiT0D4fkfneVK0fRkxMeI6mBL/LOeWOahNjczGVy+a5wfA4dfDCdw1ao8aW0zuvILkSgKfiESRcEvRKIo+IVIFAW/EImi4BciUS4o9ZnZbgB/iU4Lbgdw0N2/ZmZfAPAHABa7b/2cuz8a3VjOMDsdPt80zp6l46qtsAS0znMz4BneyisXSS4ZHeXJFAXSCqu6zmv4lWI11ercduinP6W262/mEuHx42EJKBOpdzg4wGvxZSNyaqnEpa31cljqq1a5BNuMtGwbLnE/7rnjJmorkgSjZpbXJmw1eBJO9RiX+jJrRWqbGRyhtjtuek94zDjvev/U/OvB5c0G36/N9KLzNwH8qbv/zMxGADxlZj/u2r7q7v+j560JIa4aeunVNw9gvvt6zcxeAjB3pR0TQlxZ3tY9v5ntAXAHgCe6iz5jZs+a2UNmxlvfCiGuOnoOfjMbBvB9AJ9191UAXwewF8A+dL4ZfJmMO2Bmh8zs0GqF39MJIfpLT8FvZnl0Av+b7v4DAHD3BXdvuXsbwDcA3BUa6+4H3X2/u+8fHeSVToQQ/eWCwW9mBuBBAC+5+1fOW77jvLd9DMDzl989IcSVopen/e8H8CkAz5nZM91lnwPwSTPbh478dxTAH15oRYWC4Zrd4av/mHGZ5PCxsPSysMiz8+otLg0ND/PdXq/wDLFWuxxcno2cQ5cWuYS5VuayzEaD+5F1bhsZDj96WTi1RMccX+fyVdu5RDg7zWVRa4ezy5ZXeL29gSH+mY2PcamskOXzX6sTyTfH5c31Gl9fvRxpUdbm427YvZ3adm4Pz+Ox41zSPbsYjolmrOXZJnp52v9PAEJHQFTTF0Jc3egXfkIkioJfiERR8AuRKAp+IRJFwS9EovS1gGc2ZxidIJlxRLoAgImZbNgwxIswnlngBUE3Iu2ucgVevJENazd4BmGjxf04V+Wy11Aki22jwqW56ka4gGc94mMrYnMncw+gvBpp1zUaLoQ6OsqLnVarfH1nzvK5Gh7m2YWWCV/frMll4kKOF3Ed4Io0CgU+V3tu2ENt1UrYl8cff5GOefaV0+F1bfSe1acrvxCJouAXIlEU/EIkioJfiERR8AuRKAp+IRKlr1KfmSFXDG+yOMpz/SeHw+eoXJXLaPkSz25ajfRNQ4ufD0vFmfCQPN9Wq8b72RUGuR/5HJ+PbJZLnDUP+1JvcHnTI5l7xhUxeJ1Lji1iykey6VDg8ubKMpf6qnXen25sPCzd5ogECACZyNxXwKW0hTNr1LYcyeBcWw9naf7DP77Mt0VU0Y26pD4hxAVQ8AuRKAp+IRJFwS9Eoij4hUgUBb8QidJXqa/dNpRZAcTsMB03PBTWjfIlrkMNRdKvxsa4NFde5b3kyqvhgorlSiSrb4PbRgq8AGaR9AUEgGaNS5y5XPh8Xoic5vMDPBvNjA8cjBRCzRBTs8WlqEIp0kNxnMubS0tcYlsj0ufoJJ/7SqRn4KtHeUHWl587Rm2zkzxbdHYX2bcMP06nSEHThTUue/7S6nt+pxDiXYWCX4hEUfALkSgKfiESRcEvRKJc8Gm/mRUBPA5goPv+77n7583sOgDfBrANwFMAPuXu0Ta89Tpw/I2wrbbCn86PTIefEBdLkYQOLh5gcpLvdnmd15FbWQnbls/yRJBl/nAY2TZ/yt52rmS0WlxBQDtsi53lLcMTe7I5PlfVSBKUk4f6edLGCwCaFd5SrBWp79eKJAutlMPjWBcvAFiKKD5HD/MPdOXsOrXV1/kGt4+FW3ndcu0cHcNcfPXUKh2zmV6u/DUAv+Hut6PTjvteM7sbwJcAfNXdbwCwDODTPW9VCLHlXDD4vcObHSrz3X8O4DcAfK+7/GEAH70iHgohrgg93fObWbbbofc0gB8DeA3Aivv//3J3HAD/jiKEuOroKfjdveXu+wDsAnAXgF/pdQNmdsDMDpnZoXNlXvxBCNFf3tbTfndfAfATAP8BwLiZvfk0aBeAE2TMQXff7+77x4YjHQ+EEH3lgsFvZtNmNt59XQLwmwBeQuck8Hvdt90P4EdXykkhxOWnl8SeHQAeNrMsOieL77r735jZiwC+bWb/DcDTAB680Irccmjlp4K2RmE/HVdrhxNZMs1wayoAKI5x+Wp8mn8DmcjwxJPJSjjRYmWJt3daOcPlvOo6n/5Wk8uHcH7ObjfDPm5U+S1XoRCpF5jj/q9t8MSTKrnFy0fU4JFMOFkFANoZLmE1GnweB4bCkmkxz+sFjhe4j9djnNreeztvG3bzbbdT254bbgguv+tuLm8eP1kOLv/n13hMbOaCwe/uzwK4I7D8CDr3/0KIdyD6hZ8QiaLgFyJRFPxCJIqCX4hEUfALkSjmkeyxy74xs0UAb+b1TQHoXZe4csiPtyI/3so7zY9r3X26lxX2NfjfsmGzQ+7OxX35IT/kxxX1Q1/7hUgUBb8QibKVwX9wC7d9PvLjrciPt/Ku9WPL7vmFEFuLvvYLkShbEvxmdq+Z/ZuZHTazB7bCh64fR83sOTN7xswO9XG7D5nZaTN7/rxlk2b2YzN7tfv/xBb58QUzO9Gdk2fM7CN98GO3mf3EzF40sxfM7E+6y/s6JxE/+jonZlY0s381s593/fiv3eXXmdkT3bj5jplFUj97wN37+g9AFp0yYNcDKAD4OYBb++1H15ejAKa2YLu/DuBOAM+ft+y/A3ig+/oBAF/aIj++AODP+jwfOwDc2X09AuAVALf2e04ifvR1TgAYgOHu6zyAJwDcDeC7AD7RXf6/APzRpWxnK678dwE47O5HvFPq+9sA7tsCP7YMd38cwOY61fehUwgV6FNBVOJH33H3eXf/Wff1GjrFYubQ5zmJ+NFXvMMVL5q7FcE/B+D8dqZbWfzTAfy9mT1lZge2yIc3mXX3+e7rUwBmt9CXz5jZs93bgit++3E+ZrYHnfoRT2AL52STH0Cf56QfRXNTf+D3AXe/E8BvA/hjM/v1rXYI6Jz50TkxbQVfB7AXnR4N8wC+3K8Nm9kwgO8D+Ky7v6V0Tz/nJOBH3+fEL6Fobq9sRfCfALD7vL9p8c8rjbuf6P5/GsAPsbWViRbMbAcAdP8/vRVOuPtC98BrA/gG+jQnZpZHJ+C+6e4/6C7u+5yE/NiqOelu+20Xze2VrQj+JwHc2H1yWQDwCQCP9NsJMxsys5E3XwP4LQDPx0ddUR5BpxAqsIUFUd8Mti4fQx/mxMwMnRqQL7n7V84z9XVOmB/9npO+Fc3t1xPMTU8zP4LOk9TXAPznLfLhenSUhp8DeKGffgD4FjpfHxvo3Lt9Gp2eh48BeBXAPwCY3CI//g+A5wA8i07w7eiDHx9A5yv9swCe6f77SL/nJOJHX+cEwG3oFMV9Fp0TzX8575j9VwCHAfwVgIFL2Y5+4SdEoqT+wE+IZFHwC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkyv8DgvpxjWxt2GcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = X_train[0]\n",
    "from matplotlib import pyplot as plt\n",
    "plt.imshow(data, interpolation='nearest')\n",
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define Model\n",
    "# Note: We can set some of these at variables to reduce capacity \n",
    "model = Sequential() # Sets up model structure to pass data forward \n",
    "model.add(Conv2D(32, # Activation layers \n",
    "                 (3, 3), # Conv dim\n",
    "                 padding='same', # Use padding to maintain same dims\n",
    "                 input_shape=X_train.shape[1:], activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2))) # 16x16 dim per activation\n",
    "model.add(Conv2D(32, # Activation layers \n",
    "                 (3, 3), # Conv dim\n",
    "                 padding='same', # Use padding to maintain same dims\n",
    "                 input_shape=X_train.shape[1:], activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2))) # 16x16 dim per activation\n",
    "model.add(Dropout(config.dropout)) # Add a dash of regularization (too taste)\n",
    "\n",
    "# NOTE: Would be fun to try a 16x16xActivation conv to a 1x1x10 tensor\n",
    "model.add(Flatten()) # Because dense layers like 1D vectors \n",
    "model.add(Dense(config.dense_layer_nodes, activation='relu'))\n",
    "model.add(Dropout(config.dropout)) # A pinch more regularization \n",
    "model.add(Dense(num_classes, activation='softmax')) # Plate and serve\n",
    "\n",
    "# NOTE: Could also try Adam at some point\n",
    "opt = keras.optimizers.SGD(lr=config.learn_rate) # No momentum \n",
    "\n",
    "model.compile(loss='categorical_crossentropy', # Because one hot encoding\n",
    "              optimizer=opt, # Per previous decision to not use momentum\n",
    "              metrics=['accuracy']) # Easier to look at than cross entropy\n",
    "\n",
    "# Data augmentation to improve performance on the dataset \n",
    "# datagen = ImageDataGenerator(width_shift_range=0.1)\n",
    "# datagen.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nmodel.fit_generator(datagen.flow( # Slightly clever, operate over datagen not X_train\\n                            X_train, y_train,\\n                            batch_size=config.batch_size),\\n                        # Define an epoch as a single full pass over the data\\n                        steps_per_epoch=X_train.shape[0] // config.batch_size,\\n                        # Will use a fixed set of epochs rather than a stopping criteria \\n                        epochs=config.epochs,\\n                        validation_data=(X_test, y_test),\\n                        # Allow multiple CPUs to help with data augmentation to avoid GPU starvation\\n                        workers=4,\\n                        # Make sure we can see the pastries while they bake\\n                        callbacks=[WandbCallback(data_type=\"image\", labels=class_names)]\\n)\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model on the batches generated by datagen.flow\n",
    "# For batch size make sure it's big enough to leverage GPU parallelism \n",
    "#    As a basic rule you need to reduce learning rate as you reduce batch size\n",
    "# NOTE: Will be interesting to optimize for training time a bit by tweaking batch size\n",
    "\"\"\"\n",
    "model.fit_generator(datagen.flow( # Slightly clever, operate over datagen not X_train\n",
    "                            X_train, y_train,\n",
    "                            batch_size=config.batch_size),\n",
    "                        # Define an epoch as a single full pass over the data\n",
    "                        steps_per_epoch=X_train.shape[0] // config.batch_size,\n",
    "                        # Will use a fixed set of epochs rather than a stopping criteria \n",
    "                        epochs=config.epochs,\n",
    "                        validation_data=(X_test, y_test),\n",
    "                        # Allow multiple CPUs to help with data augmentation to avoid GPU starvation\n",
    "                        workers=4,\n",
    "                        # Make sure we can see the pastries while they bake\n",
    "                        callbacks=[WandbCallback(data_type=\"image\", labels=class_names)]\n",
    ")\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "17408/50000 [=========>....................] - ETA: 2s - loss: 2.2843 - acc: 0.1350"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-58f2243654ff>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m         callbacks=[\n\u001b[1;32m     14\u001b[0m             \u001b[0mWandbCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"image\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m             \u001b[0mReduceLROnPlateau\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin_delta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.005\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         ]\n\u001b[1;32m     17\u001b[0m )\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1397\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1398\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1399\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1400\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1401\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Model \n",
    "# For batch size make sure it's big enough to leverage GPU parallelism \n",
    "#    As a basic rule you need to reduce learning rate as you reduce batch size\n",
    "# NOTE: Will be interesting to optimize for training time a bit by tweaking batch size\n",
    "model.fit(\n",
    "        X_train, \n",
    "        y_train,\n",
    "        batch_size=config.batch_size,\n",
    "        # Will use a fixed set of epochs rather than a stopping criteria \n",
    "        epochs=10,\n",
    "        validation_data=(X_test, y_test),\n",
    "        # Make sure we can see the pastries while they bake\n",
    "        callbacks=[\n",
    "            WandbCallback(data_type=\"image\", labels=class_names),\n",
    "            ReduceLROnPlateau(min_delta=0.005, patience=2)\n",
    "        ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "model2 = keras.models.load_model(\"lukas.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load(\"lukas.h5\")\n",
    "y_train_softmax_output = model.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.11243107, 0.1364868 , 0.15628915, 0.10892259, 0.14030083,\n",
       "       0.1247833 , 0.12732808, 0.12384515, 0.14750944, 0.1317587 ,\n",
       "       0.10844875, 0.15017726, 0.14847837, 0.11087765, 0.1282176 ,\n",
       "       0.14528564, 0.15323119, 0.10904116, 0.1392558 , 0.12710227,\n",
       "       0.13575622, 0.11835162, 0.10903669, 0.11636707, 0.12693058,\n",
       "       0.13882244, 0.12526631, 0.11442906, 0.12645176, 0.15872791,\n",
       "       0.12552166, 0.14409621, 0.12742051, 0.11733045, 0.12378559,\n",
       "       0.13244353, 0.13158296, 0.12182778, 0.1401626 , 0.1192603 ,\n",
       "       0.13357589, 0.13806371, 0.12602587, 0.13406932, 0.13030203,\n",
       "       0.11403947, 0.12394032, 0.11842299, 0.1312151 , 0.15390234,\n",
       "       0.11645106, 0.13566191, 0.1114352 , 0.1435345 , 0.10805386,\n",
       "       0.14622363, 0.11419181, 0.11241594, 0.11480045, 0.15236506,\n",
       "       0.13779368, 0.13385184, 0.12789743, 0.11902445, 0.12393956,\n",
       "       0.13454428, 0.11301883, 0.15417528, 0.12719199, 0.13148233,\n",
       "       0.12110427, 0.13435772, 0.13657553, 0.12534186, 0.12440172,\n",
       "       0.12403541, 0.15077858, 0.11616072, 0.1306628 , 0.14792809,\n",
       "       0.1282304 , 0.118708  , 0.12084024, 0.13910681, 0.1149644 ,\n",
       "       0.10940135, 0.11373841, 0.11372645, 0.11825161, 0.13342282,\n",
       "       0.1291189 , 0.12365592, 0.13898134, 0.12522872, 0.1466929 ,\n",
       "       0.12815173, 0.12978719, 0.12513812, 0.11696474, 0.10562007],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(y_train_softmax_output[0:100],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"y_train_softmax_output\", y_train_softmax_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH3VJREFUeJztnVuMXNd1pv9Vt67qezf7QrJJiRJ1ieRYomRG0MiejB0jgWIEkQ0Ejv1g6MEIgyAGYiB5EDzA2APMgz0Y2/DDwAN6pEQZeHyJL7EQCEkcwYGQOFBEWbLukSiKMi/NZpPdze7qqq7rmocqTaj2/jdLvFRT2v8HEKw+q/Y56+w665w656+1lrk7hBDpkdlqB4QQW4OCX4hEUfALkSgKfiESRcEvRKIo+IVIFAW/EImi4BciURT8QiRK7lIGm9m9AL4GIAvgf7v7F2Pvz+fzPlAsBm2tVouOyyD8K8Ss8W0Vcvy8lo/YctkstZmFN2gWOYdGfGw2+T7HfneZjflIfrHZ9jbfVptvzTKRHYjQbof3LeZ7dH0R/y0yycyWifiRzfDPkx0DANCO/FrWYwcCGxNdX5illTWUKxs9beyig9/MsgD+J4DfBHAcwJNm9oi7v8jGDBSL2Hfn+4K2lZUluq2BTPiDnyzwyblm2yC1TU8OUdvU+DC1FbL54PLcQImOQZZP8dLyCrXVm3zfJsbHqC3TagSX12o1OmZjY4PaiqXwyRoAWuAnr0q1HFw+Nj5Kx8D5+uq1OrVlEf5cAH6yGRnmn/PQED8+8nk+H9WIjx67QGTCx0hsn5seju8vPfh9vp3Nm+35nb/MXQAOu/sRd68D+DaA+y5hfUKIPnIpwT8H4Nh5fx/vLhNCvAO4pHv+XjCzAwAOAMDAwMCV3pwQokcu5cp/AsDu8/7e1V32Ftz9oLvvd/f9uTy/NxNC9JdLCf4nAdxoZteZWQHAJwA8cnncEkJcaS76a7+7N83sMwD+Dh2p7yF3fyE2ZmNjAy+8GH7LypkzdNwkecBq2/iT16nWCLVZaYba1ttcdSi3wk/g3Qp0TGWDP7GtVPkT+EaLS1tnIhpnMRf2sdnk68uSp81A/FatsrFObc12eL9tYxsdk4mogI2IWlHK8eOgTJ6YL7WadMzgIH/abxn+7dWIGgQAiMiHlY2wQtNshJcDQDYX/lwaG1XuwyYu6Z7f3R8F8OilrEMIsTXoF35CJIqCX4hEUfALkSgKfiESRcEvRKJc8V/4nU8GQClHZKrIj/+uJZLenlme4DIzPUltpZiUE8naqtbCCTAbDS5DeWR9hVIkISiS2ONtvr2xyXBCU7PB11fIcz8iyZbIFviHVquH56rR5PMxGFlfboj7WIyMa1pYjsxEsgSbkQy8WCbp8BBPJiuvV6it0QxLerGEyrXVc8Hl7dgHtnn9Pb9TCPGuQsEvRKIo+IVIFAW/EImi4BciUfr6tN/MUbRwQsXICHflprmJ4PJtJZ4Jkm/z0lTlJZ5s02rz82G1EvY9w/N6MBopC5aLPKVeObfGx0U+tcmR8BPntVWehFOPJOhUSdIJEK9LN0xKYTXqPPEk0+I7lo8kGLVI6TIAyJHH87UaH1PI8w800+YJQbXyMrWBJIUBwAA5jJttrkicWw8rPq1IPcbN6MovRKIo+IVIFAW/EImi4BciURT8QiSKgl+IROmr1Jczw8RAeJOliJQzRpI6pkd5zbQWaRcFINJnBsjmIoXkSB22WjsiNUV0uVwkuaRV45KYZ/k5+/TpcBegVoPv9VqFJ51UWlwWHS5Fuu/USLsu8H3OGJepsgORTjnrXNYdzId9zEVaYW1E6i5WG1zqa0earK2UuY8rlfDxUybSMgBsNMLHQD1Sq3EzuvILkSgKfiESRcEvRKIo+IVIFAW/EImi4BciUS5J6jOzowDW0FHPmu6+P7qxrGF6PCzZjOS5xFYshm2ZLJdWSpH6eI0ml73akUw197AEVI/U22vVuQzY9kjGXERi8xzPOlurhzP0Wi0+v5VIa7BmxLa2zv0/sRT2I5/h6xst87lvnOLt3KrnuFR5zdQNweUzM7voGBsJ18cDgNryWWorl3l25Lk1LvWdOReWdY8e4360suHQrdW5PLiZy6Hzf8jd+ScjhLgq0dd+IRLlUoPfAfy9mT1lZgcuh0NCiP5wqV/7P+DuJ8xsBsCPzexld3/8/Dd0TwoHAKAYua8XQvSXS7ryu/uJ7v+nAfwQwF2B9xx09/3uvr+Q012GEFcLFx2NZjZkZiNvvgbwWwCev1yOCSGuLJfytX8WwA+77a1yAP6vu/9tbEA+l8XO6XBhx9EClyiGB8PSlkWkMkQyrCySTVerctkoQ2TAbSO8bdjQEM9GWz3HRZKxUZ4xtxYpqvnGifA6yzV+y1WIJILNDUayEvM88/Do2XB2Yc0jRVcjWX1joyPUds+tXGFenQ/Lul6JbGuKZ4vWKnw+ymV+LR3I83Xu3h7et5mZWTpmYTUsHZ595RQds5mLDn53PwLg9osdL4TYWnQTLkSiKPiFSBQFvxCJouAXIlEU/EIkSn8LeGYNkyPhbLtcPSwNAcBAPuzm4EC4Lx0A1KpcDmtE+q2Nj4f7AgKAk6KP9RY/hzYakeKSw7yP38nFcC82AHjtDZ7ttbgW3rdILUhcG+l5+NH/uI/adu3g/n/vqSPB5f9ymEtRzTbPZMxluDS3trJIbZVyeB5HRrj0hhbPLiwW+bgCyT4FgEHj45qt8Idzze6ddMzIUriX47Ov87nYjK78QiSKgl+IRFHwC5EoCn4hEkXBL0Si9Pdpfy6HmcltQVt1iT8Vz1jYzTJpcwQA1Ugts5xF6tlF2lqxM2W1wZ9Sj0/wBJ16iz/BPnL8JLUtrXIfWX2/bKTF12iRr28mF36qDADFJa5I3Di6Pbh8fpL7sbBymtpqFT7HT7/yCrVlSPuqxlCk1dgYT6hBhofM2BhXn0bakfZgpM6j11fpmD0kQW4g3/v1XFd+IRJFwS9Eoij4hUgUBb8QiaLgFyJRFPxCJEqfpb48Jqamg7aJYd5eK5MJJ0WsrC7TMY31Ml9fK9auixe0c5JgNDzM6/Q1wG0vHeES1XqNt34qFge4rRD2sTTEZaiJLJdFnzq8QG3NOj98amNhqW96gs+HgctvjSaXgit1XktwndTqqzf5PltEuo10c0M+E2n1lonULsyF57FZ41KqE5mY5J4F0ZVfiERR8AuRKAp+IRJFwS9Eoij4hUgUBb8QiXJBqc/MHgLwOwBOu/uvdpdNAvgOgD0AjgL4uLtz3e3f1wYQ2c4i7YwYA5F6aoMIZz0BQC5yzstkIvX4iAw4UOLtus6c4llxlTN8yq6f5JJYjateKBJJ7+a9c3RMJrLCZpbP8WpEas1lw3UGRwr8c9k2sZfa9t54DbW9/osnqe3lV04ElxdyERnNuUzcbPKQyZCMSgDIF/g8ttvh46od0RXNwsdpRIn8JXq58v8FgHs3LXsAwGPufiOAx7p/CyHeQVww+N39cQBLmxbfB+Dh7uuHAXz0MvslhLjCXOw9/6y7z3dfn0KnY68Q4h3EJT/w804xe/qjQjM7YGaHzOzQWiVysyqE6CsXG/wLZrYDALr/0/pL7n7Q3fe7+/6RQf4QSwjRXy42+B8BcH/39f0AfnR53BFC9ItepL5vAfgggCkzOw7g8wC+COC7ZvZpAG8A+HgvG2u7o7oRLlZoDZ6ZBYQzsNbXeYHDeoOf15oZ/g2kXOHS3Cqxze3m0+hNvr5rp7gws3cnl4YqG3zc3E23B5cXnN9yLZ/jhVBL4+GCqwCAszxTbff2HcHlK+s8W/H6X7mR2kYneFbi6MQt1La8GJ7/5XO85Vk+IkdmnGdUNtqRbFGeLIpWI3x8R5IEaeu4t5HUd+Hgd/dPEtOH38Z2hBBXGfqFnxCJouAXIlEU/EIkioJfiERR8AuRKH0t4OlwtCwsh3iLF1RkskapyIt+Do9waejkIpcVXz++SG25fNiPwgLvq7exwNd34wyX8z78QS57vXZic6rFvzMyFy6QOrUtXFATAE4v8iKd4+MR2avN/S+QgpWnF8NZdgCQK65Q2+LKPLWdmOdZePl8+DgYH+XaW7XKBTPP8eulRbS5dkQGzFh4nEUyTCNtHntGV34hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkSl+lvmw2g/Hx4aCtmeNSX7kczkjzBpdPzq3xrK03fsGlrXKZy0alYvhcOf86zy6cLfKijnNz11Lb+M7rqC2/FkkRI0VNd91+Fx9yistvpSaXKlvgmYLr62HbjsGwFAkA9RbfLxsKHzcAsGtoJ7WNjIclzrWzp+iY0wtnqa1hXN7cqPOioMhwbW5oIJxlWq9GJExSENSIbBh0qed3CiHeVSj4hUgUBb8QiaLgFyJRFPxCJEpfn/a3W02srYSfpObqvNZdnrQmAi8hh1yWGytlrgRMjPBElvGh8FPZ6jJ/2j+zk9fAm7vtP1Hb88fr1PbKYW67Z8dkcPnKCh8zuzdc9w8AMqhQW73GlYBxDz+5Xz3Nn6SX6ryW4I7J8H4BwEqL19XL3zYRXF6NJAr986OPUNvxY3yfs5GWXLFGWiyPqBFrK9cIzxVLgguuo+d3CiHeVSj4hUgUBb8QiaLgFyJRFPxCJIqCX4hE6aVd10MAfgfAaXf/1e6yLwD4AwBv6h6fc/dHe9lgligerUgSgxOZJEPaeAFAy7jUt8wVJayuRuq31cJy2Y4xLg/+2oc+RG27br6b2n7w5w9R2/ZIkku2Hq5PeOLIa3x9199KbcVtN1DbkHN5trIU7t1aaoelNwCoV7mseGaN28aneRLUtu17gsur5VE6JsNNaBV4MlOshl+jwaVWa4YT1Mx54lqzGQ7dyy31/QWAewPLv+ru+7r/egp8IcTVwwWD390fB8DLxQoh3pFcyj3/Z8zsWTN7yMz4dzkhxFXJxQb/1wHsBbAPwDyAL7M3mtkBMztkZofKFX7fI4ToLxcV/O6+4O4td28D+AYAWibG3Q+6+3533z88yKvaCCH6y0UFv5ntOO/PjwF4/vK4I4ToF71Ifd8C8EEAU2Z2HMDnAXzQzPYBcABHAfxhLxszAEaUiBbJUgJ426JI5yR4NbK+SAm8yW28zdf2wbC0eOf+m+iYW+7hct7yaS5vDjR55uH1u3ZRW5vs3PYZXjuvucEl00okG7De5OMa1fCh1QKXKV87cZzannv+ELXdczf3cdv2cFbl6lpYigQA0uELADC1h8u67Vh7rXpEtiMS8rlF3r6sthZ2sk2yKUNcMPjd/ZOBxQ/2vAUhxFWJfuEnRKIo+IVIFAW/EImi4BciURT8QiRKXwt4ugNtksFUrXGJokCy2HI5XjAxm+Hyzw3b+a+RiyV+Ptxz7e7g8ts/wDP3dtx8G7U98y9/Tm3X7OY+bn/Pe6mtML03uDw3OEbHVDa45Fhd5Zl7CyePUdvyQli2azV4dl5pJFwgFQCmpvhnfezk09Q2u2MuuLxZiWSRVnnbLVtfpraWhzMqAcCZxg2gNBDet8J2vs+rAyTT9W1EtK78QiSKgl+IRFHwC5EoCn4hEkXBL0SiKPiFSJS+Sn1mhnw2vMnlSIHG1kZY1igNluiYbIZLKzORzL1j8zyTau+doVKGwK73hpd34JJdY22d2sZGuDQ3fdM+alvPhXvavfD0k3RMrcr9WF3l83HmxC+oLdsKS63FIj/k5q4Ly3IAcNtNvJBoM8sz7fLZ8fDyAs/6zG3wIp2VN05QG5OxAaAZucyWSV/JwW18v2ZJD8h8vvfrua78QiSKgl+IRFHwC5EoCn4hEkXBL0Si9Dexp91GrRp+kjo4wF2xYvhpaD7Da8h5i9tKw7yV1+/+/u9S2z2//eHg8tGpWTpm4chL1JaN+L+yxmv4LR79N2o7uRZ+4vyPf/3XdMxwiSeQbNR4Asz2Wa5IjI6En1S/fpwnA9Uj8zG5cw+13fTe91EbWgPBxUsrvF5ghahLALBc5T6a82N4o8oT18qkxZaXuepwS1jEQLv3bl268guRKgp+IRJFwS9Eoij4hUgUBb8QiaLgFyJRemnXtRvAXwKYRac910F3/5qZTQL4DoA96LTs+ri78wJnAByOtpPaem2eFGHNsEzS9EhLrkjNtOLAKLXtex+XjQbyYUnsxWd4Dbnlk69RW63GpZy15SVqO3b4RWorezjZKd/i2xrOcelztMiTS6YnuNQ3v3AquLwZactWWeOy4rHXeRIR8AK1lMvhGoTFHD8+mgMz1Ha2yY+dUonXIBwc4UlopVxYjlyrrNIxzXZYcnwbSl9PV/4mgD9191sB3A3gj83sVgAPAHjM3W8E8Fj3byHEO4QLBr+7z7v7z7qv1wC8BGAOwH0AHu6+7WEAH71STgohLj9v657fzPYAuAPAEwBm3X2+azqFzm2BEOIdQs/Bb2bDAL4P4LPu/pabEXd3kNsNMztgZofM7NB6ldfSF0L0l56C38zy6AT+N939B93FC2a2o2vfASDY8NzdD7r7fnffP1QqXA6fhRCXgQsGv5kZgAcBvOTuXznP9AiA+7uv7wfwo8vvnhDiStFLVt/7AXwKwHNm9kx32ecAfBHAd83s0wDeAPDxC6/KAYRlu3aT3xLk8uGae61IzbQ6ePbV7Bivq/d3j/wNtU3OhiWlmR3hNl4AUK/w7Lx8PizxAMDwEJeUchkuzQ0ROXL7TLjmGwBU17hCW8pyH88unqG2Rj382YwUueRVL3Op79WnD1Hb/MuvUFutSVpo5fkctmLzu4tLnxjix3BmgEutRSLbTYDP1S3vuS64vFQ8Qsds5oLB7+7/BIDlOIZzXIUQVz36hZ8QiaLgFyJRFPxCJIqCX4hEUfALkSh9LeAJN7TbYeGgEMksK+ZI8cMML7TokRZO7TrPLDtzJpyNBgDlxbCt1ODZV23w/Zqc4PLb+M5pamu2atR24mTYR4/ke2Uy/DCoN7lkmjVe+HOoGJZnSYJmZ30xYyRLs1XncmqGHG+rFS5v1geIPAhgZCef+/USb2221uYy4MZ6+Bq8bfR6OmaKSLe5fO8hrSu/EImi4BciURT8QiSKgl+IRFHwC5EoCn4hEqW/Uh8MGQtniRUHeAaTkwy9oVJYTgKAoZEpaqs0eIbVthFecyBH/KifW6Bj2hm+vkqeS1uzs+GsLQBo17lsdPNtu4LLf/qTx+iYuleoLW9cTq2W+bjRkXBWYiHHD7msRfrZbfDP7PV5LtutrIQ/s5qt0zHTN/Fr4tx4JCvR+We9fIbPVWEjLJkOzUUyMSvhrMl2RC3djK78QiSKgl+IRFHwC5EoCn4hEkXBL0Si9PVpf8aAQi58vqnUeMJElrSMakfqy1UaPDkjm+dJIgMF/jQ3nw/7URjkbavGRnmC0alFrhJU5sJP7QFgZvcN1HbidLiu3nt+7f10THnxJLUdeYW3wlov80SWXDY8/2NjvDahkfqOADB/gvv4izciiT0D4fkfneVK0fRkxMeI6mBL/LOeWOahNjczGVy+a5wfA4dfDCdw1ao8aW0zuvILkSgKfiESRcEvRKIo+IVIFAW/EImi4BciUS4o9ZnZbgB/iU4Lbgdw0N2/ZmZfAPAHABa7b/2cuz8a3VjOMDsdPt80zp6l46qtsAS0znMz4BneyisXSS4ZHeXJFAXSCqu6zmv4lWI11ercduinP6W262/mEuHx42EJKBOpdzg4wGvxZSNyaqnEpa31cljqq1a5BNuMtGwbLnE/7rnjJmorkgSjZpbXJmw1eBJO9RiX+jJrRWqbGRyhtjtuek94zDjvev/U/OvB5c0G36/N9KLzNwH8qbv/zMxGADxlZj/u2r7q7v+j560JIa4aeunVNw9gvvt6zcxeAjB3pR0TQlxZ3tY9v5ntAXAHgCe6iz5jZs+a2UNmxlvfCiGuOnoOfjMbBvB9AJ9191UAXwewF8A+dL4ZfJmMO2Bmh8zs0GqF39MJIfpLT8FvZnl0Av+b7v4DAHD3BXdvuXsbwDcA3BUa6+4H3X2/u+8fHeSVToQQ/eWCwW9mBuBBAC+5+1fOW77jvLd9DMDzl989IcSVopen/e8H8CkAz5nZM91lnwPwSTPbh478dxTAH15oRYWC4Zrd4av/mHGZ5PCxsPSysMiz8+otLg0ND/PdXq/wDLFWuxxcno2cQ5cWuYS5VuayzEaD+5F1bhsZDj96WTi1RMccX+fyVdu5RDg7zWVRa4ezy5ZXeL29gSH+mY2PcamskOXzX6sTyTfH5c31Gl9fvRxpUdbm427YvZ3adm4Pz+Ox41zSPbsYjolmrOXZJnp52v9PAEJHQFTTF0Jc3egXfkIkioJfiERR8AuRKAp+IRJFwS9EovS1gGc2ZxidIJlxRLoAgImZbNgwxIswnlngBUE3Iu2ucgVevJENazd4BmGjxf04V+Wy11Aki22jwqW56ka4gGc94mMrYnMncw+gvBpp1zUaLoQ6OsqLnVarfH1nzvK5Gh7m2YWWCV/frMll4kKOF3Ed4Io0CgU+V3tu2ENt1UrYl8cff5GOefaV0+F1bfSe1acrvxCJouAXIlEU/EIkioJfiERR8AuRKAp+IRKlr1KfmSFXDG+yOMpz/SeHw+eoXJXLaPkSz25ajfRNQ4ufD0vFmfCQPN9Wq8b72RUGuR/5HJ+PbJZLnDUP+1JvcHnTI5l7xhUxeJ1Lji1iykey6VDg8ubKMpf6qnXen25sPCzd5ogECACZyNxXwKW0hTNr1LYcyeBcWw9naf7DP77Mt0VU0Y26pD4hxAVQ8AuRKAp+IRJFwS9Eoij4hUgUBb8QidJXqa/dNpRZAcTsMB03PBTWjfIlrkMNRdKvxsa4NFde5b3kyqvhgorlSiSrb4PbRgq8AGaR9AUEgGaNS5y5XPh8Xoic5vMDPBvNjA8cjBRCzRBTs8WlqEIp0kNxnMubS0tcYlsj0ufoJJ/7SqRn4KtHeUHWl587Rm2zkzxbdHYX2bcMP06nSEHThTUue/7S6nt+pxDiXYWCX4hEUfALkSgKfiESRcEvRKJc8Gm/mRUBPA5goPv+77n7583sOgDfBrANwFMAPuXu0Ta89Tpw/I2wrbbCn86PTIefEBdLkYQOLh5gcpLvdnmd15FbWQnbls/yRJBl/nAY2TZ/yt52rmS0WlxBQDtsi53lLcMTe7I5PlfVSBKUk4f6edLGCwCaFd5SrBWp79eKJAutlMPjWBcvAFiKKD5HD/MPdOXsOrXV1/kGt4+FW3ndcu0cHcNcfPXUKh2zmV6u/DUAv+Hut6PTjvteM7sbwJcAfNXdbwCwDODTPW9VCLHlXDD4vcObHSrz3X8O4DcAfK+7/GEAH70iHgohrgg93fObWbbbofc0gB8DeA3Aivv//3J3HAD/jiKEuOroKfjdveXu+wDsAnAXgF/pdQNmdsDMDpnZoXNlXvxBCNFf3tbTfndfAfATAP8BwLiZvfk0aBeAE2TMQXff7+77x4YjHQ+EEH3lgsFvZtNmNt59XQLwmwBeQuck8Hvdt90P4EdXykkhxOWnl8SeHQAeNrMsOieL77r735jZiwC+bWb/DcDTAB680Irccmjlp4K2RmE/HVdrhxNZMs1wayoAKI5x+Wp8mn8DmcjwxJPJSjjRYmWJt3daOcPlvOo6n/5Wk8uHcH7ObjfDPm5U+S1XoRCpF5jj/q9t8MSTKrnFy0fU4JFMOFkFANoZLmE1GnweB4bCkmkxz+sFjhe4j9djnNreeztvG3bzbbdT254bbgguv+tuLm8eP1kOLv/n13hMbOaCwe/uzwK4I7D8CDr3/0KIdyD6hZ8QiaLgFyJRFPxCJIqCX4hEUfALkSjmkeyxy74xs0UAb+b1TQHoXZe4csiPtyI/3so7zY9r3X26lxX2NfjfsmGzQ+7OxX35IT/kxxX1Q1/7hUgUBb8QibKVwX9wC7d9PvLjrciPt/Ku9WPL7vmFEFuLvvYLkShbEvxmdq+Z/ZuZHTazB7bCh64fR83sOTN7xswO9XG7D5nZaTN7/rxlk2b2YzN7tfv/xBb58QUzO9Gdk2fM7CN98GO3mf3EzF40sxfM7E+6y/s6JxE/+jonZlY0s381s593/fiv3eXXmdkT3bj5jplFUj97wN37+g9AFp0yYNcDKAD4OYBb++1H15ejAKa2YLu/DuBOAM+ft+y/A3ig+/oBAF/aIj++AODP+jwfOwDc2X09AuAVALf2e04ifvR1TgAYgOHu6zyAJwDcDeC7AD7RXf6/APzRpWxnK678dwE47O5HvFPq+9sA7tsCP7YMd38cwOY61fehUwgV6FNBVOJH33H3eXf/Wff1GjrFYubQ5zmJ+NFXvMMVL5q7FcE/B+D8dqZbWfzTAfy9mT1lZge2yIc3mXX3+e7rUwBmt9CXz5jZs93bgit++3E+ZrYHnfoRT2AL52STH0Cf56QfRXNTf+D3AXe/E8BvA/hjM/v1rXYI6Jz50TkxbQVfB7AXnR4N8wC+3K8Nm9kwgO8D+Ky7v6V0Tz/nJOBH3+fEL6Fobq9sRfCfALD7vL9p8c8rjbuf6P5/GsAPsbWViRbMbAcAdP8/vRVOuPtC98BrA/gG+jQnZpZHJ+C+6e4/6C7u+5yE/NiqOelu+20Xze2VrQj+JwHc2H1yWQDwCQCP9NsJMxsys5E3XwP4LQDPx0ddUR5BpxAqsIUFUd8Mti4fQx/mxMwMnRqQL7n7V84z9XVOmB/9npO+Fc3t1xPMTU8zP4LOk9TXAPznLfLhenSUhp8DeKGffgD4FjpfHxvo3Lt9Gp2eh48BeBXAPwCY3CI//g+A5wA8i07w7eiDHx9A5yv9swCe6f77SL/nJOJHX+cEwG3oFMV9Fp0TzX8575j9VwCHAfwVgIFL2Y5+4SdEoqT+wE+IZFHwC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkyv8DgvpxjWxt2GcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = X_train[0]\n",
    "from matplotlib import pyplot as plt\n",
    "plt.imshow(data, interpolation='nearest')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2540., 7803., 9774., 8806., 7181., 6142., 4797., 2436.,  470.,\n",
       "          51.]),\n",
       " array([0.102368  , 0.10959391, 0.11681981, 0.12404573, 0.13127163,\n",
       "        0.13849755, 0.14572346, 0.15294936, 0.16017528, 0.16740118,\n",
       "        0.1746271 ], dtype=float32),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAESpJREFUeJzt3WusZeVdx/Hvr4zQVtMyyIS0M9iDdmqCl1g8Alo1WizQog4vsFIvjEicGPEajZ22L0ham0zjpZfUNiEFhUaLBFshBUsntI2aCHYoCAWsHOlUZqTl2AG0rb1M/ftiP9PsnucMM91rn9n7ON9PsrPXetaz1vqffQZ+Z631rLVTVUiSNO4Zsy5AkjR/DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1Nsy6gEmdeuqptbCwMOsyJGnduPvuu/+zqjYdTd91Gw4LCwvs2bNn1mVI0rqR5FNH2/eIp5WSXJvk8SQfH2s7JcnuJA+3942tPUnelmQpyX1JzhpbZ3vr/3CS7WPt35/k/rbO25Lk6H9USdJaOJprDn8OXLiibSdwR1VtBe5o8wAvB7a21w7gnTAKE+Aq4BzgbOCqQ4HS+vzK2Hor9yVJOsaOGA5V9XfAgRXN24Dr2vR1wMVj7dfXyJ3AyUmeB1wA7K6qA1X1BLAbuLAte05V3Vmjx8NeP7YtSdKMTDpa6bSqeqxNfxo4rU1vBh4d67evtT1d+75V2iVJMzR4KGv7i/+YfClEkh1J9iTZs7y8fCx2KUnHpUnD4TPtlBDt/fHWvh84fazfltb2dO1bVmlfVVVdXVWLVbW4adNRjcaSJE1g0nC4BTg04mg7cPNY+2Vt1NK5wFPt9NPtwPlJNrYL0ecDt7dl/5Xk3DZK6bKxbUmSZuSI9zkkeQ/wY8CpSfYxGnW0C7gxyRXAp4BXtu63Aa8AloAvAJcDVNWBJG8APtr6vb6qDl3k/jVGI6KeBfxte0mSZijr9TukFxcXy5vgJOnoJbm7qhaPpu+6vUNa35iFnbfOZL97d100k/1KGsYH70mSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOn6HtNbUrL67Gvz+amkIjxwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSZ1B4ZDkd5I8kOTjSd6T5JlJzkhyV5KlJH+V5MTW96Q2v9SWL4xt5zWt/RNJLhj2I0mShpo4HJJsBn4TWKyq7wZOAC4F3gS8uapeCDwBXNFWuQJ4orW/ufUjyZltve8CLgTekeSESeuSJA039LTSBuBZSTYAzwYeA14K3NSWXwdc3Ka3tXna8vOSpLXfUFVfqqpPAkvA2QPrkiQNMHE4VNV+4I+Af2cUCk8BdwNPVtXB1m0fsLlNbwYebesebP2/dbx9lXW+TpIdSfYk2bO8vDxp6ZKkIxhyWmkjo7/6zwCeD3wzo9NCa6aqrq6qxapa3LRp01ruSpKOa0NOK/0E8MmqWq6qrwDvBV4CnNxOMwFsAfa36f3A6QBt+XOBz463r7KOJGkGhoTDvwPnJnl2u3ZwHvAg8GHgktZnO3Bzm76lzdOWf6iqqrVf2kYznQFsBf5pQF2SpIE2HLnL6qrqriQ3AR8DDgL3AFcDtwI3JPmD1nZNW+Ua4N1JloADjEYoUVUPJLmRUbAcBK6sqq9OWtc8W9h566xLkKSjMnE4AFTVVcBVK5ofYZXRRlX1ReBnDrOdNwJvHFKLJGl6vENaktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnUEP3pPm2ayegrt310Uz2a80TR45SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqTMoHJKcnOSmJP+S5KEkP5jklCS7kzzc3je2vknytiRLSe5LctbYdra3/g8n2T70h5IkDbNh4PpvBT5QVZckORF4NvBa4I6q2pVkJ7ATeDXwcmBre50DvBM4J8kpwFXAIlDA3UluqaonBtYmzcTCzltnst+9uy6ayX71/9PERw5Jngv8KHANQFV9uaqeBLYB17Vu1wEXt+ltwPU1cidwcpLnARcAu6vqQAuE3cCFk9YlSRpuyGmlM4Bl4M+S3JPkXUm+GTitqh5rfT4NnNamNwOPjq2/r7Udrl2SNCNDwmEDcBbwzqp6MfB5RqeQvqaqitGpoqlIsiPJniR7lpeXp7VZSdIKQ8JhH7Cvqu5q8zcxCovPtNNFtPfH2/L9wOlj629pbYdr71TV1VW1WFWLmzZtGlC6JOnpTBwOVfVp4NEk39mazgMeBG4BDo042g7c3KZvAS5ro5bOBZ5qp59uB85PsrGNbDq/tUmSZmToaKXfAP6ijVR6BLicUeDcmOQK4FPAK1vf24BXAEvAF1pfqupAkjcAH239Xl9VBwbWJUkaYFA4VNW9jIagrnTeKn0LuPIw27kWuHZILZKk6fEOaUlSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHU2zLoASdOxsPPWme17766LZrZvrQ2PHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQZHA5JTkhyT5L3t/kzktyVZCnJXyU5sbWf1OaX2vKFsW28prV/IskFQ2uSJA0zjSOH3wIeGpt/E/Dmqnoh8ARwRWu/Aniitb+59SPJmcClwHcBFwLvSHLCFOqSJE1oUDgk2QJcBLyrzQd4KXBT63IdcHGb3tbmacvPa/23ATdU1Zeq6pPAEnD2kLokScMMPXJ4C/D7wP+2+W8Fnqyqg21+H7C5TW8GHgVoy59q/b/Wvso6kqQZmDgckvwk8HhV3T3Feo60zx1J9iTZs7y8fKx2K0nHnSFHDi8BfjrJXuAGRqeT3gqcnOTQN8xtAfa36f3A6QBt+XOBz463r7LO16mqq6tqsaoWN23aNKB0SdLTmTgcquo1VbWlqhYYXVD+UFX9PPBh4JLWbTtwc5u+pc3Tln+oqqq1X9pGM50BbAX+adK6JEnDrcV3SL8auCHJHwD3ANe09muAdydZAg4wChSq6oEkNwIPAgeBK6vqq2tQlyTpKE0lHKrqI8BH2vQjrDLaqKq+CPzMYdZ/I/DGadQiSRrOO6QlSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLU2TDrAmZhYeetsy5BkuaaRw6SpM5xeeQgabpmdTS+d9dFM9nv8cAjB0lSx3CQJHUMB0lSx3CQJHUMB0lSZ+JwSHJ6kg8neTDJA0l+q7WfkmR3kofb+8bWniRvS7KU5L4kZ41ta3vr/3CS7cN/LEnSEEOOHA4Cv1tVZwLnAlcmORPYCdxRVVuBO9o8wMuBre21A3gnjMIEuAo4BzgbuOpQoEiSZmPicKiqx6rqY236v4GHgM3ANuC61u064OI2vQ24vkbuBE5O8jzgAmB3VR2oqieA3cCFk9YlSRpuKtcckiwALwbuAk6rqsfaok8Dp7XpzcCjY6vta22Ha19tPzuS7EmyZ3l5eRqlS5JWMTgcknwL8NfAb1fVf40vq6oCaug+xrZ3dVUtVtXipk2bprVZSdIKg8IhyTcxCoa/qKr3tubPtNNFtPfHW/t+4PSx1be0tsO1S5JmZMhopQDXAA9V1Z+MLboFODTiaDtw81j7ZW3U0rnAU+300+3A+Uk2tgvR57c2SdKMDHnw3kuAXwTuT3Jva3stsAu4MckVwKeAV7ZltwGvAJaALwCXA1TVgSRvAD7a+r2+qg4MqEuSNNDE4VBV/wDkMIvPW6V/AVceZlvXAtdOWoskabq8Q1qS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1Nkw6wIkaVILO2+d2b737rpoZvs+FjxykCR1DAdJUsdwkCR1DAdJUsdwkCR15iYcklyY5BNJlpLsnHU9knQ8m4twSHIC8KfAy4EzgVclOXO2VUnS8WsuwgE4G1iqqkeq6svADcC2GdckScetebkJbjPw6Nj8PuCcGdUiSUc0qxvwjtXNd/MSDkclyQ5gR5v9XJJPrOhyKvCfx7aqiVjndFnndK2XOmH91Dq1OvOmQau/4Gg7zks47AdOH5vf0tq+TlVdDVx9uI0k2VNVi9Mvb7qsc7qsc7rWS52wfmpdL3WOm5drDh8FtiY5I8mJwKXALTOuSZKOW3Nx5FBVB5P8OnA7cAJwbVU9MOOyJOm4NRfhAFBVtwG3DdzMYU85zRnrnC7rnK71Uiesn1rXS51fk6qadQ2SpDkzL9ccJElzZF2Ew5EerZHkR5N8LMnBJJesWLY9ycPttX2O6/xAkieTvH8taxxaa5LvS/KPSR5Icl+Sn53TOl/Q2u9ttf7qPNY5tvw5SfYlefu81pnkq+3zvDfJmg4YGVjntyX5YJKHkjyYZGHe6kzy42Of5b1Jvpjk4rWqcyJVNdcvRheo/w34duBE4J+BM1f0WQC+F7geuGSs/RTgkfa+sU1vnLc627LzgJ8C3j/nn+mLgK1t+vnAY8DJc1jnicBJbfpbgL3A8+etzrHlbwX+Enj7PP7e27LPrfW/zSnV+RHgZWO/+2fPY51jfU4BDqxVnZO+1sORwxEfrVFVe6vqPuB/V6x7AbC7qg5U1RPAbuDCOayTqroD+O81qm2liWutqn+tqofb9H8AjwOb5rDOL1fVl9rsSaztUfKg332S7wdOAz64hjUOrvMYmrjO9ky2DVW1u/X7XFV9Yd7qXOES4G/XsM6JrIdwWO3RGpuPwbrfqGO5r6GmUmuSsxn9xfRvU6prpUF1Jjk9yX1tG29qYbYWJq4zyTOAPwZ+bw3qWmno7/2ZSfYkuXONT4EMqfNFwJNJ3pvkniR/2B7suRam9d/8pcB7plLRFK2HcNAcSvI84N3A5VU1y78yD6uqHq2q7wVeCGxPctqsa1rFrwG3VdW+WRdyFF5Qo7t8fw54S5LvmHVBq9gA/AijsP0BRqd8fmmWBT2d9t/R9zC6x2uurIdwOKpHa6zBut+oY7mvoQbVmuQ5wK3A66rqzinXNm4qn2k7Yvg4o/9prIUhdf4g8OtJ9gJ/BFyWZNd0y/uaQZ9nVe1v748wOq//4mkWN2ZInfuAe9upnoPA3wBnTbm+Q6bx7/OVwPuq6itTq2pK1kM4DHm0xu3A+Uk2JtkInM/aJfR6egTIxLW2/u8Drq+qm9awRhhW55Ykz2rTG4EfBlY+qHHmdVbVz1fVt1XVAqO/dq+vqrX6sqshn+fGJCe16VOBlwAPzludbd2Tkxy6DvZS5rPOQ17FHJ5SAuZ/tFKNrua/AvhXRue2X9faXg/8dJv+AUZ/MXwe+CzwwNi6vwwstdflc1zn3wPLwP+0PhfMY63ALwBfAe4de33fHNb5MuA+RiNI7gN2zOPnuWIbv8QajlYa+Hn+EHB/+zzvB66YxzpX/O7vB/4cOHFO61xgdKTxjLX8LCd9eYe0JKmzHk4rSZKOMcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktT5P3S+6H9ajodjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "maxes = np.max(y_train_softmax_output,axis=1)\n",
    "plt.hist(maxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
